package supermarket;
import java.io.BufferedWriter;
import java.io.FileWriter;
import java.io.IOException;
import java.net.MalformedURLException;
import java.util.Iterator;
import java.util.List;

import org.apache.commons.logging.LogFactory;

import com.gargoylesoftware.htmlunit.BrowserVersion;
import com.gargoylesoftware.htmlunit.FailingHttpStatusCodeException;
import com.gargoylesoftware.htmlunit.NicelyResynchronizingAjaxController;
import com.gargoylesoftware.htmlunit.ScriptResult;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.html.DomAttr;
import com.gargoylesoftware.htmlunit.html.DomNode;
import com.gargoylesoftware.htmlunit.html.DomText;
import com.gargoylesoftware.htmlunit.html.HtmlAnchor;
import com.gargoylesoftware.htmlunit.html.HtmlDivision;
import com.gargoylesoftware.htmlunit.html.HtmlForm;
import com.gargoylesoftware.htmlunit.html.HtmlListItem;
import com.gargoylesoftware.htmlunit.html.HtmlOption;
import com.gargoylesoftware.htmlunit.html.HtmlPage;
import com.gargoylesoftware.htmlunit.html.HtmlRadioButtonInput;
import com.gargoylesoftware.htmlunit.html.HtmlSelect;
import com.gargoylesoftware.htmlunit.html.HtmlSubmitInput;
import com.gargoylesoftware.htmlunit.html.HtmlTextInput;
import com.gargoylesoftware.htmlunit.html.HtmlUnorderedList;
import com.gargoylesoftware.htmlunit.javascript.host.Event;


public class HeadlessBrowser {
	private BufferedWriter bufferedWriter;
	
	public static void main(String[] args) throws FailingHttpStatusCodeException, MalformedURLException, IOException{
		LogFactory.getFactory().setAttribute("org.apache.commons.logging.Log", "org.apache.commons.logging.impl.NoOpLog"); 
		final WebClient webClient = new WebClient();
//		IncLis incLis = new IncLis();
//		webClient.setIncorrectnessListener(incLis);
//		System.out.println(webClient.isJavaScriptEnabled());

	    // Get the first page
		// selecteer taal NL
	    HtmlPage page = (HtmlPage) webClient.getPage("http://www.delhaizedirect.be/");
	    HtmlAnchor anchor = page.getAnchorByHref("javascript:__doPostBack('linkNl','')");
	    page = (HtmlPage) anchor.click();
//	    System.out.println(page.getTitleText());

	    //****deze pauze is niet altijd nodig, hangt af van de laadtijd van delhaizedirect****
	    try {
			Thread.sleep(5000);
		} catch (InterruptedException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		
	    // form selecteren
	    HtmlForm form = page.getFormByName("aspnetForm");

	    //winkel selecteren
	    final HtmlSelect selectShop = form.getSelectByName("ctl00$ctl00$mainContent$mainColumnContent$ShopList$ddShopList");
	    final HtmlOption optionToSelect = selectShop.getOptionByValue("10014014");
//	    final HtmlOption optionToSelect = selectShop.getOptionByValue("10014037");
	    selectShop.focus();
	    page = (HtmlPage) selectShop.setSelectedAttribute(optionToSelect, true);
//	    System.out.println(selectShop.getOnChangeAttribute());
//	    ScriptResult result = selectShop.fireEvent(Event.TYPE_CHANGE);
//	    page = (HtmlPage) result.getNewPage();
//	    page = optionToSelect.click();
	    
	    // sleep 5s to wait for response of asynchronous call
	    try {
			Thread.sleep(5000);
		} catch (InterruptedException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
//	    System.out.println(page.asXml());
	    
		// selecteren van afhaaltijdstip
	    HtmlRadioButtonInput radio = page.getFormByName("aspnetForm").getRadioButtonsByName("SlotGroup").get(0);
	    
//	    page = (HtmlPage) radio.setChecked(true);
	    page = radio.click();

	    try {
	    	Thread.sleep(3000);
	    } catch (InterruptedException e) {
	    	// TODO Auto-generated catch block
	    	e.printStackTrace();
	    }
//	    System.out.println(page.getFormByName("aspnetForm").getRadioButtonsByName("SlotGroup").get(0).isChecked());
	    
	    final HtmlAnchor button = (HtmlAnchor) page.getElementById("ctl00_ctl00_mainContent_mainColumnContent_linkStart");
//	    final HtmlAnchor button = (HtmlAnchor) page.getAnchorByHref("javascript:__doPostBack('ctl00$ctl00$mainContent$mainColumnContent$linkStart','')");
	    page = button.click();
	    
	    try {
			Thread.sleep(3000);
		} catch (InterruptedException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
//	    System.out.println(page.asXml());
	    
	    //itereren
	    
	    int level = 1;
//	    boolean isFirstLevel = (level == 1);
	    
	    String levelString = "level_0" + level;
	    // /html/body/form[@name='aspnetForm']/div[@id='container']/div[@class='content']/div[@id='menu']/ul[@class='level_01']
	    HtmlUnorderedList levelList = page.getFirstByXPath("//ul[@class='" + levelString + "']");
	    //alle items van level_01
	    //TODO levelList geeft soms nullpointer 
	    List<HtmlListItem> levelItems = (List<HtmlListItem>) levelList.getByXPath(".//li");
	    Iterator<HtmlListItem> it = levelItems.iterator();
	    
		HeadlessBrowser hb = new HeadlessBrowser();
	    //itereren over items level_01
	    while(it.hasNext()){
	    	HtmlListItem item = it.next();
		    HtmlAnchor link = item.getFirstByXPath(".//a[@class='middleLink']");
		    HtmlPage pageWithLinks = link.click();
//		    isFirstLevel = false;
			try {
				Thread.sleep(3000);
			} catch (InterruptedException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
	    	}

			hb.scrapeRecursively(1,pageWithLinks);
			System.out.println("een item van level_01 gedaan");
	    }

	    
//	    webClient.closeAllWindows();
	}
	public HeadlessBrowser(){
		initializeWriter();
	}
	private void initializeWriter(){
		try {
			bufferedWriter = new BufferedWriter(new FileWriter("delhaize.txt"));
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}

	}
	private void scrapePage(HtmlPage page){
	    // scrapen van productinformatie
	    List<HtmlDivision> allProducts = (List<HtmlDivision>) page.getByXPath("//div[@class='detailProduct']");
	    
	    for(int i = 0; i < allProducts.size(); i++){
		    HtmlDivision product = allProducts.get(i);
//		    System.out.println(allProducts.get(i).asXml());
		    DomText productBrand = (DomText) product.getFirstByXPath("./div[@class='detailWrapper']//a[@class='PB_Product_Brand'][1]/text()");
//		    System.out.println(page.getByXPath("//div[@class='detailWrapper']//a[@class='PB_Product_Brand'][1]/text()").size()); 
		    DomAttr productName  = (DomAttr) product.getFirstByXPath("./div[@class='detailWrapper']//a[@class='PB_Product_Name'][1]/@title");
		    DomAttr href  = (DomAttr) product.getFirstByXPath("./div[@class='detailWrapper']//a[@class='PB_Product_Name'][1]/@href");
		    
		    String productID = href.getValue();
		    String[] productIDparts = productID.split("pid=");
		    productID = productIDparts[1].split("&")[0];
		    
		    DomText productDisplayPrice = (DomText) product.getFirstByXPath("./div[@class='priceListWrapper']//span[@class='price PB_Product_DisplayPrice'][1]/text()");
		    DomText productWeight = (DomText) product.getFirstByXPath("./div[@class='priceListWrapper']//span[@class='productListPackaging PB_Product_Packaging'][1]/text()");
		    DomText productBorderPrice = (DomText) product.getFirstByXPath("./div[@class='priceListWrapper']//span[@class='borderPrice'][1]/text()");
		    writeLine("PID: " + productID + " merk: " + productBrand + " productnaam: " + productName.getValue() + " displayprijs: " + productDisplayPrice + 
		    		" gewicht: " + productWeight + " prijs per kilo: " + productBorderPrice);
		    System.out.println("PID: " + productID + " merk: " + productBrand + " productnaam: " + productName.getValue() + " displayprijs: " + productDisplayPrice + 
		    		" gewicht: " + productWeight + " prijs per kilo: " + productBorderPrice);
	    }
	}
	private void writeLine(String line){
		try {
			bufferedWriter.write(line);
			bufferedWriter.newLine();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
	}
	public void scrapeRecursively(int level, DomNode itemToScrape) throws IOException{
    	int nextLevel = level + 1;
    	String levelString = "level_0" + nextLevel;
	    HtmlUnorderedList nextLevelList = itemToScrape.getFirstByXPath("//ul[@class='" + levelString + "']");
		
	    if(nextLevelList != null){
	    	List<HtmlListItem> items = (List<HtmlListItem>) nextLevelList.getByXPath(".//li");
	    	Iterator<HtmlListItem> it = items.iterator();
	    	while(it.hasNext()){
	    		HtmlListItem item = it.next();
	    		HtmlUnorderedList unorderedList = item.getFirstByXPath("descendant::ul");
	    		if(unorderedList == null){
	    	    	HtmlAnchor linkToPage = item.getFirstByXPath(".//a");
	    	    	//tijdelijk filteren op W07003003010006
//	    	    	System.out.println(linkToPage.asXml());
	    	    	if(
//	    	    			!linkToPage.asXml().contains("W01") &&
//	    	    			!linkToPage.asXml().contains("W02") && 
//	    	    			!linkToPage.asXml().contains("W03") &&
//	    	    			!linkToPage.asXml().contains("W04") &&
//	    	    			!linkToPage.asXml().contains("W05") &&
//	    	    			!linkToPage.asXml().contains("W06") &&
//	    	    			!linkToPage.asXml().contains("W07") &&
//	    	    			!linkToPage.asXml().contains("W08") &&
//	    	    			!linkToPage.asXml().contains("W09") &&
//	    	    			!linkToPage.asXml().contains("W10") &&
//	    	    			!linkToPage.asXml().contains("W11") &&
//	    	    			!linkToPage.asXml().contains("W12") &&
//	    	    			!linkToPage.asXml().contains("W13") &&
	    	    			!linkToPage.asXml().contains("W07003003010006") && 
	    	    			!linkToPage.asXml().contains("W07007005002") && 
	    	    			!linkToPage.asXml().contains("W07008002") &&
	    	    			!linkToPage.asXml().contains("W07009004001") &&
	    	    			!linkToPage.asXml().contains("W11005") &&
	    	    			!linkToPage.asXml().contains("W11007005") &&
	    	    			!linkToPage.asXml().contains("W12003002") &&
	    	    			!linkToPage.asXml().contains("W12003001") &&
	    	    			!linkToPage.asXml().contains("W12002001") &&
	    	    			!linkToPage.asXml().contains("W12002002") &&
	    	    			!linkToPage.asXml().contains("W12001001") &&
	    	    			!linkToPage.asXml().contains("W13004006") &&
	    	    			!linkToPage.asXml().contains("W15002005")){

//	    	    		W07008002001, W07008002002, W11005001, W11005003, W11005002, W11007005, W12003002, W12003001, W12002001
//	    	    		W12002002, W12001001, W13004006, W15002005
	    	    	
	    	    		HtmlPage pageToScrape = linkToPage.click();
	    	    		try {
	    	    			Thread.sleep(5000);
	    	    		} catch (InterruptedException e) {
	    	    			// TODO Auto-generated catch block
	    	    			e.printStackTrace();
	    	    		}
	    	    		scrapePage(pageToScrape);
	    	    	}
	    	    	else{
	    	    		System.out.println("Er werd een categorie genegeerd");
	    	    	}
	    	    }
		    	flush();
	    	}

	    }
	}
	private void flush() {
		try {
			bufferedWriter.flush();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
	}
}
